#!/usr/bin/env python
# -*- coding:utf-8 -*-
# author:sirian
# datetime:2018/11/8 15:23
# software: PyCharm
'''
#找到糗事百科最热门链接的
#用户名

#图片链接（非用户的）

#段子的文字内容

#点赞数量

#评论数量

'''
import urllib2
import random
import json
from lxml import etree

url = "https://www.qiushibaike.com/8hr/page/1/"

ua = {
    "Connection": "keep-alive",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36",
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
    "Accept-Language": "zh-CN,zh;q=0.9",
}


#是否使用代理
proxy_list=[
#{"http" : "221.2.155.35:8060"}#最好用刚刚出的免费代理时间
]
if proxy_list:
    proxy_ip = random.choice(proxy_list)
    http_handler = urllib2.ProxyHandler(proxy_ip)
else:
    http_handler = urllib2.ProxyHandler({})

opener = urllib2.build_opener(http_handler)
request = urllib2.Request(url, headers=ua)
response = opener.open(request)

#处理文本
content_html = response.read()

content_xml = etree.HTML(content_html)

# 返回所有段子的结点位置，contains()模糊查询方法，第一个参数是要匹配的标签，第二个参数是标签名部分内容
node_list = content_xml.xpath('//div[contains(@class, "article block untagged mb15")]')
user_all_information = {}
for node in node_list:
    # 用户名
    username = node.xpath('.//img/@alt')
    # 图片链接（非用户的）,有的客户没有
    user_send_img = node.xpath('.//div[contains(@class ,"thumb")]/a/img/@src')
    # 段子的文字内容
    user_content = node.xpath('.//div[contains(@class, "content")]/span')
    # 点赞数量
    user_zan = node.xpath('.//div[contains(@class, "stats")]/span[@class="stats-vote"]/i[@class="number"]')
    # 评论数量
    user_pinglun = node.xpath('.//div[contains(@class ,"stats")]//span[@class="stats-comments"]//i[@class="number"]')
    #print user_zan
    #print(user_zan)
    user_all_information = {
        "username" : username[0],
        "user_send_img" : user_send_img,
        "user_content" : user_content[0].text,#text取下标签半酣的内容，因为不是@的
        "user_zan" : user_zan[0].text,
        "user_pinglun" : user_pinglun[0].text
    }
    #print user_all_information["user_zan"]
    user_all = json.dumps(user_all_information, ensure_ascii=False)
    #
    with open("qiushibaike.json", "a") as f:
        f.write(user_all.encode("utf-8")+"\n")



